import scrapy
from kickstart import items
import json
import re
import copy
import pickle
from  bson.binary import Binary

class CtripSpider(scrapy.Spider):
    name = "ctripyouji"
    allowed_domains = ["hotels.ctrip.com", "images4.c-ctrip.com",
    "you.ctrip.com"]
    start_urls = ["http://you.ctrip.com/"]
    prefix = 'http://you.ctrip.com/TravelSite/Home/IndexTravelListHtml?p=%s&Idea=0&Type=2&Plate=0'
    testurl = 'http://you.ctrip.com/travels/hailuogou705/2041634.html'
    def __init__(self, spiderid=None, step=None, cityfilepath=None, *args, **kwargs):
        super(CtripSpider, self).__init__(*args, **kwargs)
    
    def parseYouji(self, response):
        youji = items.CtripYouji()
        youji['user'] = response.meta['user']
        youji['date'] = response.meta['date']
        youji['place'] = response.meta['place']
        youji['title'] = response.meta['title']
        content = ""
        #datas = response.selector.xpath('/html/body/div[3]/div[4]/div[1]/div[1]/div[2]/p/text()').extract()
        #datas = response.selector.xpath('//div[@class="bgf2f2f2"]/div[4]/div[1]/div[1]/div[2]/*')
        datas = response.selector.xpath('//div[@class="ctd_content wtd_content"]/* | //div[@class="ctd_content"]/*')
        #print "xxxxxxxxxxxxxxxxxxxxxxxxxx",datas
        for data in datas:
            tmp = data.xpath('text() | a/text()').extract()
            for t in tmp:
                content += t;
        youji['content'] = content
        youji['url'] = response.url
        yield youji
        print youji

    def parseNext(self, response):
        pid = response.meta['pid']
        if response.body:
            print "get data for page id: ", pid
            travels = response.selector.xpath('/html/body/div')
            for travel in travels:
                user = travel.xpath('div[2]/p/a[2]/text()').extract()
                date = travel.xpath('div[2]/p/i/text()').extract()
                place = travel.xpath('div[1]/a[1]/@href').extract()
                title = travel.xpath('div[1]/a[2]/text()').extract()
                turl = "http://you.ctrip.com" + travel.xpath('div[1]/a[2]/@href').extract()[0]
                yield scrapy.Request(turl, self.parseYouji, meta={'user': user, 'date': date, 'place': place, 'title': title})
            url = self.prefix % str(pid+1)
            yield scrapy.Request(url, self.parseNext, meta={'pid': pid+1})
        else:
            print "empty data for: ", response.meta['pid']

    def parse(self, response):
        url = 'http://you.ctrip.com/TravelSite/Home/IndexTravelListHtml?p=1&Idea=0&Type=2&Plate=0'
        yield scrapy.Request(url, callback=self.parseNext, meta={'pid': 1})
        #yield scrapy.Request(self.testurl, callback=self.parseYouji, meta={'pid': 1})
